package org.net;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.swing.text.EditorKit;
import javax.swing.text.SimpleAttributeSet;
import javax.swing.text.html.HTML;
import javax.swing.text.html.HTMLDocument;
import javax.swing.text.html.HTMLEditorKit;

import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.net.alexa.AlexaSeeker;
import org.net.pr.GooglePageRank;
import org.net.util.ComplexExportExcelClient;
import org.net.util.FileUtil;
import org.net.util.StringUtil;
import org.net.vo.SiteVo;

public class NetUtil {
	
	public static Long getTargetStr(String urlStr, String charset) {
		long start = System.currentTimeMillis();
        URL url = null;
        HttpURLConnection httpConn = null;
        InputStream in = null;
        String cookie = "";
        try {
                do {
                        if (StringUtil.isEmpty(urlStr)) {
                                return 1000000L;
                        }

                        url = new URL(urlStr);
                        httpConn = (HttpURLConnection) url.openConnection();
                        HttpURLConnection.setFollowRedirects(true);
                        httpConn.setRequestMethod("GET");
                        httpConn.setConnectTimeout(10000);
                        httpConn.setRequestProperty("User-Agent",
                                        "Mozilla/4.0 (compatible; MSIE 6.0; Windows 2000)");
                        if (cookie.length() != 0)
                                httpConn.setRequestProperty("Cookie", cookie);
                        httpConn.setInstanceFollowRedirects(false);
                        int code = httpConn.getResponseCode();
                        if (code == HttpURLConnection.HTTP_MOVED_TEMP) {
                                cookie += httpConn.getHeaderField("Set-Cookie") + ";";
                        }
                        if (httpConn.getResponseCode() == HttpURLConnection.HTTP_OK)
                                break;
                } while (true);
                in = httpConn.getInputStream();
                BufferedReader br = new BufferedReader(new InputStreamReader(in,
                                StringUtil.isEmpty(charset) ? "gb2312" : charset));
                String line = "";
                StringBuffer sbf = new StringBuffer();
                while ((line = br.readLine()) != null) {
                        sbf.append(line);
                }
        } catch (MalformedURLException e) {
                e.printStackTrace();
        } catch (IOException e) {
                e.printStackTrace();
        } finally {
                try {
                        in.close();
                        httpConn.disconnect();
                } catch (Exception ex) {
                        ex.printStackTrace();
                }
        }
        long end =System.currentTimeMillis();
        return end - start;
}
	

	public static Long getHTML(String url,SiteVo sv) throws Exception {
		try{
		long start = System.currentTimeMillis();
		HttpClient httpClient = new HttpClient();
		httpClient.getHostConfiguration().setProxy("127.0.0.1", 8087);
		List<Header> headers = new ArrayList<Header>();   
        headers.add(new Header("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)"));   
        httpClient.getHostConfiguration().getParams().setParameter("http.default-headers", headers);   
		GetMethod getMethod = new GetMethod(url);
		getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,   
                new DefaultHttpMethodRetryHandler(0, false));
//		getMethod.getParams().setContentCharset("gb2312");
		httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(20000);  
        getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT,20000);  
        System.err.println("Ready to crawl："+url);
		int statusCode = httpClient.executeMethod(getMethod);
		if (statusCode != HttpStatus.SC_OK) {
			System.err.println("Method failed: " + getMethod.getStatusLine());
			return null;
		}
		
		// 读取内容
//		InputStream   txtis   =   getMethod.getResponseBodyAsStream();
//		BufferedReader br = new BufferedReader(new InputStreamReader(txtis));
//
//		String tempbf;
//		StringBuffer html=new StringBuffer();
//		while((tempbf=br.readLine())!=null){
//		html.append(tempbf);
//		} 
		
		String responseBody = getMethod.getResponseBodyAsString(); 
//		String responseBody = html.toString();
		if(!StringUtil.isEmpty(responseBody) && ((StringUtil.ignoreIndexOf(responseBody,"If you have a TypeKey or TypePad account, please Sign In",0,true)!=-1)
				|| StringUtil.ignoreIndexOf(responseBody,"If you have a TypeKey or TypePad account, please Sign In",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Comment below or sign in with TypePad Facebook Twitter and more",0,true)!=-1
				|| StringUtil.ignoreIndexOf(responseBody,"There may be a delay before your comment is published",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Displayed next to your comments. Post a new comment",0,true)!=-1
				|| StringUtil.ignoreIndexOf(responseBody,"Comments by Joomla Professional Solutions",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Powered by !JoomlaComment",0,true)!=-1
				|| StringUtil.ignoreIndexOf(responseBody,"Please note, comments will be published immediately",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Enter YourName@YourKeywords in the Name field to take advantage",0,true)!=-1
				|| StringUtil.ignoreIndexOf(responseBody,"Powered by: BlogEngine.NET",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"To help keep spam off our site, please enter the letter",0,true)!=-1
				|| StringUtil.ignoreIndexOf(responseBody,"Empty lines separate paragraphs. Use @code@ for inline code, and > code for code blocks",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"You must read and type the 5 chars within 0..9 and A..F, and submit the form",0,true)!=-1
				|| StringUtil.ignoreIndexOf(responseBody,"Your comment was successfully posted",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"This question is for testing whether you are a human visitor and to prevent automated spam submissions",0,true)!=-1
				|| StringUtil.ignoreIndexOf(responseBody,"blog comments powered by DISQUS Add New Comment",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Spam Protection by WP-SpamFree Plugin LEAVE A REPLY",0,true)!=-1
				|| StringUtil.ignoreIndexOf(responseBody,"Powered by Movable Type Pro",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Spam Protection by WP-SpamFree",0,true)!=-1
				|| StringUtil.ignoreIndexOf(responseBody,"Spam Protection by RDV",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Leave a comment",0,true)!=-1
				|| StringUtil.ignoreIndexOf(responseBody,"Leave a Reply",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Post your response",0,true)!=-1
				|| StringUtil.ignoreIndexOf(responseBody, "Post a comment", 0, true)!=-1) && StringUtil.ignoreIndexOf(responseBody, "Sign in to comment",0,true)==-1){
			sv.setHasIdentidy("true");
		}else{
			sv.setHasIdentidy("false");
		}
		getMethod.releaseConnection();
	    long end = System.currentTimeMillis();
		return end - start;
		}catch(Exception e){
			e.printStackTrace();
			sv.setHasIdentidy("exception");
			return 50000L;
		}
	}
	
	 public static String getHTML2(String url) throws Exception {  
         HttpClient httpClient = new HttpClient();  
         httpClient.getHostConfiguration().setProxy("127.0.0.1", 8087);
 		 List<Header> headers = new ArrayList<Header>();   
         headers.add(new Header("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)"));   
         httpClient.getHostConfiguration().getParams().setParameter("http.default-headers", headers); 
         GetMethod getMethod = new GetMethod(url);
         System.err.println("Ready to crawl："+url);
//         getMethod.getParams().setContentCharset("gb2312");
         getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,   
                 new DefaultHttpMethodRetryHandler(3, false));
         httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(20000);  
         getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT,20000);  
         int statusCode = httpClient.executeMethod(getMethod);  
         if (statusCode != HttpStatus.SC_OK) {  
             System.err.println("Method failed: " + getMethod.getStatusLine());  
             return null;  
         }  
         // 读取内容  
         String responseBody = getMethod.getResponseBodyAsString();  
         getMethod.releaseConnection();  
         return responseBody;  
   
     } 

	/**
	 * 
	 * @throws Exception
	 * 
	 * 经测试用getHTML2 方法正则查找最快
	 * getHTML 和  getTargetStr 都比较慢
	 */
	public static void test(String url,String sheetName) throws Exception {
		
        long start = System.currentTimeMillis();
		String html = getHTML2(url);
		Pattern p = null;
		Matcher m = null;
		StringBuffer sb0 = new StringBuffer();
		
		String regex ="<span class=\"result\"><a.*href=\"(.*)\".*>(.+?)</a>(.*)<p class=\"details\">(.*)</p><p class=\"info\">(.*)<span class=\"size\">(.*)</span></p></span>";
		
		/**
		// 链接正则
		String regexa = "<a.*href=\"(.*)\".*>(.+?)</a>(.*)<p class=\"details\">(.*)</p><p class=\"info\">(.*)<span class=\"size\">(.*)</span></p>";
		p = Pattern.compile(regex);
		m = p.matcher(html);
		int count = 0;
		while (m.find()) {
			sb0.append(m.group());
		}
		p = Pattern.compile(regexa);
		m = p.matcher(sb0.toString());
		// 链接地址和标题
		while (m.find()) {
			System.out.println("地址:" + m.group(1));
			System.out.println("标题:" + m.group(2));
			System.out.println("时间:" + m.group(3));
			count++;
		}
        **/
		int count =0 ;
		int validCount =0;
		p = Pattern.compile(regex);
		m = p.matcher(html);
		while (m.find()) {
			sb0.append(m.group());
		}
		
		List<SiteVo> array = new ArrayList<SiteVo>();
		
		EditorKit kit = new HTMLEditorKit();
	    HTMLDocument doc = (HTMLDocument) kit.createDefaultDocument();
	    kit.read(new ByteArrayInputStream(sb0.toString().getBytes()), doc, 0);

	    HTMLDocument.Iterator it = doc.getIterator(HTML.Tag.A);
	    while (it.isValid()) {
	      SimpleAttributeSet s = (SimpleAttributeSet) it.getAttributes();

	      String link = (String) s.getAttribute(HTML.Attribute.HREF);
	      String title = (String) s.getAttribute(HTML.Attribute.TITLE);
	      if (link != null && !StringUtil.isEmpty(title) && link.equalsIgnoreCase(title)) {
//	        System.out.println(validCount+"："+link);
	        validCount++;
	        SiteVo site = new SiteVo();
	        site.setUrl(link);
	        array.add(site);
	      }
	      count ++;
	      it.next();
	    }
	    
//	    int spanCount =0;
	    //String spanRegex ="<p class=\"info\"><span class=\"type\">text/html</span>(.*)<span class=\"size\">(.*)</span></p>";
//	    String spanRegex ="<span class=\"size\">\\d+k</span>";
//	    p = Pattern.compile(spanRegex);
//		m = p.matcher(sb0.toString());
		// 链接地址和标题
//		while (m.find()) {
////			System.out.println(spanCount+"页面大小:" + m.group().replaceAll("<span class=\"size\">","").replaceAll("</span>", ""));
//			array.get(spanCount).setCount(m.group().replaceAll("<span class=\"size\">","").replaceAll("</span>", "").replaceAll("k", ""));
//			spanCount++;
//		}
		
		int index =1;
//		Collections.sort(array,new Comparator<SiteVo>(){
//			 public int compare(SiteVo o1, SiteVo o2){
//			      return Integer.parseInt(o1.getCount()) - Integer.parseInt(o2.getCount());
//			 }
//		});
		Long itsTime =0L;
		List<SiteVo> newarray = new ArrayList<SiteVo>();
		for(SiteVo sv:array){
			if(sv.getUrl().length()>=35){  //简单根据url判断是否能留言
			itsTime = getHTML(sv.getUrl(),sv);
			sv.setOpenedTime(itsTime);
			GooglePageRank prService = new GooglePageRank();
			sv.setPrvalue(String.valueOf(prService.getPageRank(sv.getUrl())));
			AlexaSeeker a = new AlexaSeeker();
			sv.setAlexavalue(a.getValues("POPULARITY", "TEXT",sv.getUrl()));
//			if(!StringUtil.isEmpty(sv.getAlexavalue()) && Long.parseLong(sv.getAlexavalue())<1000000){ //alexa排名100W以内的
//			 if(!StringUtil.isEmpty(sv.getPrvalue()) && Integer.parseInt(sv.getPrvalue())>1){ //pr值大于1的
			   if("true".equalsIgnoreCase(sv.getHasIdentidy())){
			      System.out.println(index+"："+sv.getUrl()+"----"+sv.getCount()+"---opened:"+sv.getOpenedTime()+"---pr:"+sv.getPrvalue()+"---alexa:"+sv.getAlexavalue()+"---hasIdentity："+sv.getHasIdentidy());
			      index++;
			      newarray.add(sv);
			   }
//			}
//			}
			}
		}
		
		ComplexExportExcelClient source = new ComplexExportExcelClient();
		source.ExportSiteInfo(sheetName, newarray);
		System.out.println("指定区域内共找到链接数：" + count+"，其中可用链接："+(--index));
		long end = System.currentTimeMillis();
		System.out.println("耗时: "+(end-start));
		FileUtil.appendContentByRandomFile("d:\\","crawledUrl.txt", url+",\r\n");
	}
	
	  //重点在于正则表达式 <img.*src=(.*?)[^>]*?>     
	  //               src=\"?(.*?)(\"|>|\\s+)   
	public static List<String> getImgStr(String htmlStr){     
	     String img="";     
	     Pattern p_image;     
	     Matcher m_image;     
	     List<String> pics = new ArrayList<String>();  
	  
	     String regEx_img = "<img.*src=(.*?)[^>]*?>"; //图片链接地址     
	     p_image = Pattern.compile   
	             (regEx_img,Pattern.CASE_INSENSITIVE);     
	     m_image = p_image.matcher(htmlStr);   
	     while(m_image.find()){     
	         img = img + "," + m_image.group();     
	         Matcher m  = Pattern.compile("src=\"?(.*?)(\"|>|\\s+)").matcher(img); //匹配src  
	         while(m.find()){  
	            pics.add(m.group(1));  
	         }  
	     }     
	        return pics;     
	 }    
	
	public static void test() throws Exception{  
        
		String url ="http://cpc.people.com.cn/GB/194302/194306/index.html";
        String html = getHTML2(url);  
        Pattern p = null;  
        Matcher m = null;  
        StringBuffer sb0 = new StringBuffer();  
        // ul正则  
        String regex = "<ul class=\"d2_9\">([\\s\\S]*<li>)<a.*href='(.*)'.*>(.+?)</a> \\[(.*)\\]</li>([\\s].*)";  
        // 链接正则  
        String regexa = "<a.*href='(.*)'.*>(.+?)</a> \\[(.*)\\]";  
        p = Pattern.compile(regex);  
        // m = p.matcher(sb.toString());  
        m = p.matcher(html);  
        int count = 0;  
        // ul字符串  
        while (m.find()) {  
            sb0.append(m.group());  
        }  
        //System.out.println(sb0.toString());  
        p = Pattern.compile(regexa);  
        m = p.matcher(sb0.toString());  
        // 链接地址和标题  
        while (m.find()) {  
            System.out.println("地址:" + m.group(1));  
            System.out.println("标题:" + m.group(2));  
            System.out.println("时间:" + m.group(3));  
            count++;  
        }  
          
        System.out.println("抓取条数："+count);  
  
    } 

	public static void main(String[] args) throws Exception {
		//String url = "http://siteexplorer.search.yahoo.com/search?p=www.parpoo.com&fr=sfp&bwm=i";
		//test(url);
		
		String str = FileUtil.readTextFile("d:\\", "source.txt");
		String[] urls = str.split(",");
		for(String url:urls){
			if(!StringUtil.isEmpty(url)){
			final String target = url;
			new Thread(){
				public void run(){
					try {
						test("http://siteexplorer.search.yahoo.com/search?p="+target+"&fr=sfp&bwm=i",target);
					} catch (Exception e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}
				}
			}.start();
			}
		}
		
	}
}

